package paper.parse;

import java.util.Vector;
import java.util.Iterator;
import java.util.regex.*;
import java.io.*;

import paper.bayes.*;
import paper.token.TT;
import paper.token.Token;
import paper.parse.TokenInputStream;

public class TokenStream implements TokenInputStream {
    Vector<Token> tokens = new Vector<Token>();

    boolean pending = false;
    boolean isItalic = false, pendingIsItalic = false;
    boolean isBold   = false, pendingIsBold   = false;
    float size = 0, pendingSize = 0;
    boolean pendingIsFont = false, pendingNewLine = true;
    public StringBuilder sb = new StringBuilder();
    public StringBuilder inputBuffer = new StringBuilder();
    public String finalString;
    private int start = 0;
    private int endOfTitleBlock, beginningOfBib, endOfFirstPage = -1;
    private PrintWriter textFile;
    
    static final Bayes b = new Bayes(new File("lib/authors.hash"),
                                     new File("lib/titles.hash"),
                                     new File("lib/affiliations.hash"),
                                     new File("lib/wheres.hash"));

    public TokenStream(PrintWriter text) {
        textFile = text;
    }

    
    final static Pattern rubbishPatt = Pattern.compile("(a[0-9]+a[0-9]+a[0-9]+(a[0-9]+ ?)+)|(g[0-9]+g[0-9]+g[0-9]+(g[0-9]+ ?)+)|(c[0-9]+c[0-9]+c[0-9]+(c[0-9]+ ?)+)");


    final static Pattern absPatt = Pattern.compile("(\n;? *abstract(\\.)? *\n)|(\n;? *abstract[^:] .*)|(\n(1.? +(; +)?)? *introduction(\\.)? *\n)", Pattern.CASE_INSENSITIVE);
    final static Pattern bibPatt = Pattern.compile("\n *([1-9][0-9]*\\.?)?( ?;?) *(reference(s?)|bibliography)(:|;|\\.)? *\n", Pattern.CASE_INSENSITIVE);

    final static String brk = "(\\b|\\B)";

    final static String monthAbbr = "(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\\.?";
    final static String monthFull = "(january|february|march|april|may|june|july|august|september|october|november|december)";
    final static String month = "(((" + monthAbbr + ")|(" + monthFull + ")),? *)";
    final static String year = "(((19[0-9][0-9])|(20[0-9][0-9]))[a-g]?)";
    final static String day = "(([1-9]|[12][0-9]|3[01])(st|nd|rd|th)?)";
    final static String spacing = "[ \n]*";
    final static Pattern datePatt =
        Pattern.compile( brk + "((" + month + spacing + day + ",?" + spacing + year + ")" + "|" +
                         "(" + day + spacing + month + spacing + year + ")" + "|" +
                         "(" + month + spacing + year + ")" + "|" +
                         "(" + month + spacing + day + "-" + day + spacing + year + ")" + "|" +
                         "(c ?" + year + ")" + "|" +
                         "(" +  year + ")" + ")" + brk, Pattern.CASE_INSENSITIVE );

    final static String dc = "[-a-z0-9_#]+";
    final static String domain = dc + "(\\." + dc + ")+";
    final static String username = "[-a-z0-9_\\.#,|]+";
    final static Pattern emailPatt = 
        Pattern.compile( brk + "(e-?mail:? *)?((\\{" + username + "([\n ]+" + username + ")*\\}@" + domain + ")" + "|" +
                         "(" + username + "@" + domain + "))" + brk, Pattern.CASE_INSENSITIVE );



    final static Pattern pagePatt =
        Pattern.compile( brk + "(pages|pp\\.?)? +[1-9][0-9]*[-?{][1-9][0-9]*" + brk );

    final static Pattern numberPatt =
        Pattern.compile( brk + "[1-9][0-9][-,#():/0-9a-zA-Z]+" );



    final static String volumeWithVol = "((vol\\.?|volume) *[1-9][0-9]* *(\\([1-9][0-9]*(-[1-9][0-9]*)?\\))?)";
    final static String volumeWithParentheses = "[1-9][0-9]*\\([1-9][0-9]*(-[1-9][0-9]*)?\\)";
    final static Pattern volumePatt =
        Pattern.compile(brk + "(" + volumeWithVol + "|" +
                        volumeWithParentheses +")[:,\\. ]",
                        Pattern.CASE_INSENSITIVE);

    private int replaceTokens(int startToken, int start, int end, TT newTokenType) {
//        System.out.println("Start  " + start + " end " + end + " replace with " + newTokenType);
        for (int i = startToken; i < tokens.size(); i++) {
            Token t = tokens.get(i);
//            if (t.from > start - 10 && t.from < start + 10) {
//                System.out.println("From " + t.from + " " + start + " end " + t.length +" " + end + " replace " + t.t + " with " + newTokenType);
//            }
//            System.out.println("TRYING " + i);
            if (t.from >= start) {
                while (t.t == TT.EOL) {
//                    System.out.println("SKIPPING EOL " + i);
                    ++i;
                    if (i < tokens.size()) {
                        t = tokens.get(i);
                    } else {
                        break;
                    }
                }
                if(t.t != TT.EMAIL && t.t != TT.NUMBER && t.t != TT.EOL) {
                    t.lengthOfWord = 0;
                    t.t = newTokenType;
                    ++i;
//                    System.out.println("Setting token to " + t);
                    while (i < tokens.size()) {
//                        System.out.println("MODDING " + i +" " + tokens.size());
                        Token nextT = tokens.get(i);
                        if (nextT.from < end) {
                            t.length += nextT.length;
                            tokens.remove(i);
                        } else {
                            return i;
                        }
                    }
                }
                return i;
            }
        }
        return tokens.size();
    }

    private int distill(TT newTokenType, Pattern p) {
        int startToken = 0;
        Matcher m = p.matcher(finalString);
        m.region(0, endOfTitleBlock);
        while (m.find()) {
//            System.out.println("Found a match for " + finalString.substring(m.start(), m.end()));
            startToken = replaceTokens(startToken, m.start(), m.end(), newTokenType);
        }
        m.region(beginningOfBib, finalString.length());
        while (m.find()) {
//            System.out.println("Found a match for " + finalString.substring(m.start(), m.end()));
            startToken = replaceTokens(startToken, m.start(), m.end(), newTokenType);
        }
        return -1;
    }

    public void eof() {
        finalString = sb.toString();
        textFile.print(finalString);
        textFile.close();

        Matcher m = absPatt.matcher(finalString);
        if (m.find()) {
            endOfTitleBlock = m.start();
            if (endOfTitleBlock > 1500) {
                endOfTitleBlock = 1500;
            }
        } else {
            endOfTitleBlock = 1500;
            if (endOfTitleBlock > finalString.length()) {
                endOfTitleBlock = finalString.length();
            }
        }
        if (endOfTitleBlock > endOfFirstPage) {
            endOfTitleBlock = endOfFirstPage;
        }
        m = bibPatt.matcher(finalString);
        if (m.find()) {
            beginningOfBib = m.start();
            if (beginningOfBib < finalString.length()/3) {
                beginningOfBib = finalString.length();
            }
        } else {
            beginningOfBib = finalString.length();
        }
//        beginningOfBib = finalString.length();   // to stop bib

        distill(TT.EMAIL,  emailPatt);
        distill(TT.NUMBER, volumePatt);
        distill(TT.NUMBER, pagePatt);
        distill(TT.NUMBER, datePatt);
        distill(TT.NUMBER, numberPatt);

        int nUnknown = 0, nKnown = 0;
        for (int i = 0; i < tokens.size(); i++) {
            Token t = tokens.get(i);
            if (t.from >= endOfTitleBlock && t.from <= beginningOfBib) {
                continue;    // skip core of paper.
            }
//            System.out.println("Looking at " + t);
            String clean = t.getCleanText();
            if (!clean.equals("") && t.t == TT.UNKNOWN) {
                t.probs = new double[TT.nTT];
                int x = b.classify(clean, t.probs);
                t.t = (x == 0 ? TT.AUTHOR :
                       x == 1 ? TT.TITLE :
                       x == 2 ? TT.AFFILIATION :
                       x == 3 ? TT.WHERE : TT.UNKNOWN);
                if (t.t == TT.UNKNOWN) {
                    nUnknown++;
                    t.probs = null;
                } else {
//                    System.out.println("        " + clean + " " + t.probs[0] + " " + t.probs[1] + " "+ t.probs[2] + " "+ t.probs[3]);
                    nKnown++;
                    boolean reNormalise = false;
                    double sum = 1.0;
                    final double notBase = 0.01;
                    final double base    = 1.0 - notBase*(TT.nTT-1);
                    final double noClue  = 1.0/TT.nTT;
                    for (int k=0; k<TT.nTT; k++) {                
                        if (t.probs[k] < notBase) {
                            sum += notBase - t.probs[k];
                            t.probs[k] = notBase;
                            reNormalise = true;
                        }
                    }
                    if (sum == 0) {
                        System.err.println("SUM SHOULD NOT BE ZERO");
                    }
                    if (reNormalise) {
                        for (int k=0; k<TT.nTT; k++) {     
                            t.probs[k] /= sum;
                        }
                    }
                }
            }
        }
        System.out.println("Knowns " + nKnown + " / " + nUnknown);
        if (nKnown * 3 < nUnknown) {   // 1 known on every 3 unknown: rubbish
            titleTokens = new Vector<Token>();
        } else {
            gobble();
        }
    }

    public boolean matches(Token start, Pattern p) {
        Matcher m = p.matcher(finalString);
        if (m.find(start.from)) {
            if (m.start() == start.from) {
                return true;
            }
        }
        return false;
    }

    public void addCharacter(String s) {
        if (pending) {
            processWords();
        }
        inputBuffer.append(s);
    }
    

    double pendingXOffset = 0;
    double pendingYSpacing = 0;

    public void newPage() {
        processWords();
        if (endOfFirstPage == -1) {
            endOfFirstPage = sb.length();
        }
    }

    public void newParagraph() {
//        newLine(0);
    }

    public void newLine(double xOffset, double ySpacing) {
        if (inputBuffer.length() > 0 &&
            inputBuffer.charAt(inputBuffer.length()-1) == '-') {
            inputBuffer.deleteCharAt(inputBuffer.length()-1);
        } else {
            pendingNewLine = true;
            pendingXOffset = xOffset;
            pendingYSpacing = ySpacing;
            pending = true;
            inputBuffer.append("\n");
            processWords();
        }
    }

    public void setXOffset(double xOffset) {
//        System.out.println("New page on " + xOffset);
        newLine(xOffset, 0);
/*        pendingXOffset = xOffset;
        pending = true;
        inputBuffer.append("\n");
        processWords();*/
    }

    public void newWord() {
        inputBuffer.append(" ");
    }

    public void newGap() {
        if (sb.length() < 1500) {
            inputBuffer.append("; ");
        }
    }

    private int decodeHash(StringBuilder input, int j, StringBuilder into) {
        if (j < input.length() - 3) {
            if (input.charAt(j+1) == '0') {
                switch (input.charAt(j+2)) {
                case 'B': into.append("ff"); return j+2;
                case 'C': into.append("fi"); return j+2;
                case 'D': into.append("fl"); return j+2;
                case 'E': into.append("ffi"); return j+2;
                }
            } else if (input.charAt(j+1) == '1') {
                switch (input.charAt(j+2)) {
                case '3': return j+2; // some accent
                case '4': return j+2; // upside down halfcircle
                case '5': return j+2; // another upside down halfcircle
                }
            } else if (input.charAt(j+1) == '2') {
                switch (input.charAt(j+2)) {
                case '3': return decodeHash(input,j+2,into); // meta #2328 etc
                case '8': into.append("("); return j+2;
                case '9': into.append(")"); return j+2;
                case 'F': into.append("/"); return j+2;
                }
            } else if (input.charAt(j+1) == '5') {
                switch (input.charAt(j+2)) {
                case 'B': into.append("["); return j+2;
                case 'C': into.append("\""); return j+2;
                case 'D': into.append("]"); return j+2;
                }
            } else if (input.charAt(j+1) == '7') {
                switch (input.charAt(j+2)) {
                case 'B': into.append("-"); return j+2;
                case 'F': return j+2;                     // trema
                }
            }
        }
        return -1;
    }

    private void cleanLigatures(StringBuilder input, StringBuilder into) {
        for (int j = 0; j < input.length(); j++) {
            String ligature = null;
            switch (input.codePointAt(j)) {
            case 11:ligature = "ff"; break;
            case 12:ligature = "fi"; break;
            case 13:ligature = "fl"; break;
            case 14:ligature = "ffi"; break;
            }
            if (ligature != null) {
                into.append(ligature);
                continue;
            }
            char c = input.charAt(j);
            
            if (c == '#') {
                int next = decodeHash(input, j, into);
                if (next != -1) {
                    j = next;
                    continue;
                }
            }
            if (Character.getType(c) != Character.MODIFIER_LETTER && 
                Character.getType(c) != Character.MODIFIER_SYMBOL) {
                if (c == '\n' || c >= ' ' && c <= '}') {
                    into.append(c);
                }
            }
        }
    }

    boolean atNewLine = false;

    public void processWords() { 
//        System.out.println("Buff " + inputBuffer);
        Matcher m = rubbishPatt.matcher(inputBuffer.toString());
        while (m.find()) {
            int start = m.start(), end = m.end();
            for (int i = start; i<end; i++) {
                inputBuffer.setCharAt(i,'.');
            }
        }
        if (atNewLine) {
            while (inputBuffer.length() > 0 && inputBuffer.charAt(0) == ' ') {
                inputBuffer.deleteCharAt(0);
            }
        }
        cleanLigatures(inputBuffer, sb);
        do {
            int startOfWord = start;
            for (; start < sb.length(); start++) {
                char c = sb.charAt(start);
                if (!Character.isLetterOrDigit(c)) {
                    break;
                }
            }
            int endOfWord = start;
            for (; start < sb.length(); start++) {
                char c = sb.charAt(start);
                if (Character.isLetterOrDigit(c)) {
                    break;
                }
            }
            int endOfRest = start;
            if (startOfWord != endOfRest &&
                !(startOfWord + 1 == endOfRest && sb.charAt(startOfWord) == '\n')) {
                tokens.add(new Token(this, startOfWord, endOfRest - startOfWord,
                                     endOfWord - startOfWord, TT.UNKNOWN));
                atNewLine = false;
            }
        } while (start < sb.length());
        inputBuffer = new StringBuilder();

        if (pending) {
            if (pendingNewLine) {
                pendingNewLine = false;
                atNewLine = true;
                tokens.add(new Token(this,start, 0, 0, TT.EOL, pendingXOffset, pendingYSpacing));
            }
            if (pendingIsFont) {
                pendingIsFont = false;
                tokens.add(new Token(this,start, 0, 0, TT.FONTCHANGE));
            }
            if (size != pendingSize) {
//                System.out.println("Size " + pendingSize);
                if (pendingSize > 13) {
                    tokens.add(new Token(this,start, 0, 0, TT.LARGE));
                } else {
                    tokens.add(new Token(this,start, 0, 0, TT.NOTLARGE));
                }
                size = pendingSize;
            }
            if (isBold != pendingIsBold) {
                if (pendingIsBold) {
                    tokens.add(new Token(this,start, 0, 0, TT.BOLD));
                } else {
                    tokens.add(new Token(this,start, 0, 0, TT.NOTBOLD));
                }
                isBold = pendingIsBold;
            }
            if (isItalic != pendingIsItalic) {
                if (pendingIsItalic) {
                    tokens.add(new Token(this,start, 0, 0, TT.ITALIC));
                } else {
                    tokens.add(new Token(this,start, 0, 0, TT.NOTITALIC));
                }
                isItalic = pendingIsItalic;
            }
            pending = false;
        }
    }

    private void setPending() {
        pending =
            pendingIsBold != this.isBold || 
            pendingIsItalic != this.isItalic ||
            pendingSize != size;
    }

    public void newFont() {
//        System.out.println("-- NEW FONT");
        pendingIsFont = true;
        pending = true;
        processWords();
   }

    public void newBold(boolean isBold) {
        pendingIsBold = isBold;
        setPending();
   }

    public void newItalic(boolean isItalic) {
        pendingIsItalic = isItalic;
        setPending();
    }

    public void newSize(float size) {
        pendingSize = size;
        setPending();
    }

    public Vector<Token> bibList() {
        int i;
        for (i = 0; i < tokens.size(); i++) {
            Token t = tokens.get(i);
            if (t.from >= beginningOfBib) {
                break;
            }
        }
        return new Vector<Token>(tokens.subList(i, tokens.size()));
    }



    Vector<Token> titleTokens;


    private void gobble() {
        titleTokens = new Vector<Token>();
        int i = 0;
        while (true) {
            Token t;
            boolean dot = false;
            double probs[] = {1,1,1,1};
            int l = 0;
            int from = 0;
            do {
                t = tokens.get(i);
//                System.out.println("= " + t + "\t" + probs[0] + " " + probs[1] + " " + probs[2] + " " + probs[3]);
                if (t.t == TT.EMAIL || t.t == TT.EOL) {
                    break;
                }
                if (l == 0) {
                    from = t.from;
                }
                if (t.probs != null) {
                    for(int j = 0; j < 4; j++) {
                        probs[j] *= t.probs[j];
                    }
                } else if (t.t == TT.NUMBER) {
                    probs[0] *= 0.2;
                    probs[1] *= 0.05;
                    probs[2] *= 0.2;
                    probs[3] *= 0.5;
                }
                String r = t.getRawText();
                if (r.indexOf(".") != -1) {
                    probs[0] *= 0.39;
                    probs[1] *= 0.005;
                    probs[2] *= 0.21;
                    probs[3] *= 0.39;
                }
                l += t.length;
                i++;
            } while (t.from < endOfTitleBlock);
            if (l != 0) {
                double sum = 0;
                int maxJ = -1;
                double maxProb = 0;
                for(int j = 0; j < 4; j++) {
                    sum += probs[j];
                    if (probs[j] > maxProb) {
                        maxProb = probs[j];
                        maxJ = j;
                    }
                }
                for(int j = 0; j < 4; j++) {
                    probs[j] /= sum;
                }
                maxProb /= sum;
                if (maxProb < 0.26) maxJ = -1;
                Token newT = new Token(this, from, l, l,
                                       maxJ == 0 ? TT.AUTHOR :
                                       maxJ == 1 ? TT.TITLE :
                                       maxJ == 2 ? TT.AFFILIATION :
                                       maxJ == 3 ? TT.WHERE : TT.UNKNOWN);
                newT.probs = probs;
                titleTokens.add(newT);
//                System.out.println(probs[0] +"  " + probs[1] + " " + probs[2] + " " + probs[3] + " " + maxJ + " " +newT);
            }
            if (t.t == TT.EMAIL) {
                titleTokens.add(t);
//                System.out.println("+ " + t);
            }
            if (t.from >= endOfTitleBlock) {
                break;
            }
            i++;
        }
//        System.out.println("+-+-+- EOF");
    }











    // Methods to read from the Token stream

    int index = 0;

    public boolean atEnd() {
        return index >= titleTokens.size();
    }

    public Token next() {
        return titleTokens.get(index++);
    }

    public int mark() {
        return index;
    }

    public void reset(int mark) {
        index = mark;
    }

    public int length() {
        return titleTokens.size();
    }

    public String substring(int from, int to) {
        return sb.substring(from,to);
    }

    public String substring(int from) {
        return sb.substring(from);
    }

    // Convenience method

    public String toString() {
        StringBuilder b = new StringBuilder(1000);
        boolean first = true;
        for (Iterator<Token> i = titleTokens.iterator(); i.hasNext();) {
            Token t = i.next();
            if (t.from >= endOfTitleBlock && t.from <= beginningOfBib) {
                if (first) {
                    first = false;
                    b.append("<b><font size=+2>[...und so weiter...]</font></b>");
                }
                continue;    // skip core of paper.
            }
            b.append(t.toString());
        }
        return b.toString();
    }


}
